﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using ReBasic.Model;
using ReBasic.Helper;
using ReBasic.Database;

namespace ReBasic.Algorithm
{
    public class SeedPairExtractor
    {
        public event MainFrame.InfoHandler StateEvent;
        public event MainFrame.TripleHandler ProcessEvent;
        public event MainFrame.ProcessHandler DetailProcessEvent;

        StreamReader sr;
        StreamWriter sw;
        FileStream fsr, fsw;
        //ProBaseHeper pbh;

        List<Triple_hrt> hrt_set;
        List<string> Target_Class_Set;
        List<Quad_hrtT>[] QuadSet;
        double[] QuadSet_Sum;

        bool[,] Relevant;
        double[,] Matrix;   //Matrix[i,j] : T(i|j)
        double[,] Q_Class_Tuple;    //Record array
        int[] TcForTuple;           //Record the TcIdx for each candidate tuple.
        //Q_Class_Tuple[i,j]: the P(T|t) generated by the i-th Class and j-th tuple

        const int M1 = 1000;
        const int M2 = 1000;

        int TcIdx = -1;
        string Target_Class;

        #region EventHandler

        public void OnDetailProcessEvnet(int perc)
        {
            if (DetailProcessEvent != null)
                DetailProcessEvent(perc);
        }

        public void OnStateEvent(string text, bool isOver)
        {
            if (StateEvent != null)
                StateEvent(text, isOver);
        }

        public void OnProcessEvent(int va, int vb, int perc)
        {
            if (ProcessEvent != null)
                ProcessEvent(va, vb, perc);
        }

        #endregion

        #region Get Candidate Class Set
        double LCSR(string attr, string sup)
        {
            int i, j;
            int la = attr.Length, lb = sup.Length;
            int[,] dp = new int[la + 1, lb + 1];
            for (i = 1; i <= la; i++)
                for (j = 1; j <= lb; j++)
                    if (attr[i - 1] == sup[j - 1])
                        dp[i, j] = dp[i - 1, j - 1] + 1;
                    else
                        dp[i, j] = Math.Max(dp[i - 1, j], dp[i, j - 1]);
            return 1.0 * dp[la, lb] / Math.Max(la, lb);
        }

        void FindClass_Similarity(string attr, Dictionary<string, int> SupConDict)
        {
            int cnt = 0, i;
            List<SortInstance> tmplst = new List<SortInstance>();
            OnStateEvent("\tSelecting By Method 1...", false);
            foreach (KeyValuePair<string, int> kvp in SupConDict)
            {
                double value = LCSR(attr, kvp.Key);
                tmplst.Add(new SortInstance(kvp.Key, value));
                if (++cnt % 10000 == 0)
                    OnProcessEvent(MainFrame.TotalSupConcept, cnt,
                        (int)(100.0 * cnt / MainFrame.TotalSupConcept));
            }
            tmplst.Sort();
            int len = tmplst.Count;
            for (i = 0; i < M1; i++)
                Target_Class_Set.Add(tmplst[i].instance);
            OnStateEvent("\tSelecting Complete.", true);
        }

        double Probabilistic_Kernel(List<string> te_lst, string sup, int[] p_t)
        {
            int flag = 0, len = te_lst.Count, i;
            double ret = 0.0;
            Dictionary<string, int> rel_dict = new Dictionary<string,int>();
            //List<Relation> rel_set = pbh.FindSuperConcept(sup, 1000);
            List<IndexRelation> ixrel_set = ProBaseInterface.FindTopInSup(sup, 1000);
            foreach (IndexRelation ixrel in ixrel_set)
                rel_dict.Add(ProBase.SubUhash[ixrel.sub_idx], ixrel.freq);
            //int fc = pbh.FindSuperCount(sup, true);
            int fc = ProBaseInterface.SupFreq(sup);
            for (i = 0; i < len; i++)
            {
                string te = te_lst[i];
                if (rel_dict.ContainsKey(te))
                {
                    flag++;
                    if (flag == 1)
                        ret = 1.0;
                    ret = ret * rel_dict[te] * MainFrame.TotalFrequency / fc / p_t[i];
                }
            }
            ret = ret * fc / MainFrame.TotalFrequency;
            return ret;
        }

        void FindClass_Probabilistic(string attr, Dictionary<string, int> SupConDict)
        {
            int te_cnt = 0, cnt = 0, i, len;
            Dictionary<string, int> tmpidx = new Dictionary<string,int>();
            List<SortInstance> tmplst = new List<SortInstance>();
            List<string> te_lst = new List<string>();
            string[] te_set = new string[1000];
            double[] te_value = new double[1000];
            OnStateEvent("\tSelecting By Method 2", false);
            #region Get Top target entities
            foreach (Triple_hrt tp in hrt_set)
            {
                string s = tp.host_entity;
                if (!tmpidx.ContainsKey(s))
                    tmpidx.Add(s, te_cnt++);
                int idx = tmpidx[s];
                if (te_value[idx] < tp.q)
                {
                    te_value[idx] = tp.q;
                    te_set[idx] = tp.target_entity;
                }
            }
            for (i = 0; i < te_cnt; i++)
                te_lst.Add(te_set[i]);
            #endregion
            len = te_lst.Count;
            int[] p_t = new int[len];
            for (i = 0; i < len; i++)
                p_t[i] = ProBaseInterface.SubFreq(te_lst[i]);
            foreach (KeyValuePair<string, int> kvp in SupConDict)
            {
                double value = Probabilistic_Kernel(te_lst, kvp.Key, p_t);
                tmplst.Add(new SortInstance(kvp.Key, value));
                if (++cnt % 10000 == 0)
                    OnProcessEvent(MainFrame.TotalSupConcept, cnt,
                        (int)(100.0 * cnt / MainFrame.TotalSupConcept));
            }
            tmplst.Sort();
            len = tmplst.Count;
            for (i = 0; i < M1; i++)
                Target_Class_Set.Add(tmplst[i].instance);
            OnStateEvent("\tSelecting Complete....", true);
        }

        void GetTargetClassSet(string attr, Dictionary<string, int> SupConDict)
        {
            FindClass_Probabilistic(attr, SupConDict);
            FindClass_Similarity(attr, SupConDict);
            Target_Class_Set = Target_Class_Set.Distinct().ToList();
            OnStateEvent("\tGet " + Target_Class_Set.Count.ToString() + " dist. TCs.", true);
        }
        #endregion

        #region Traversal Triple
        public void GenMatrix()
        {
            int i, C, T;
            int len = Target_Class_Set.Count, len_tuple = hrt_set.Count;
            OnStateEvent("\tGenerating Matrix...", false);
            Matrix = new double[len, len];
            TcForTuple = new int[len_tuple];
            Q_Class_Tuple = new double[len, len_tuple];
            QuadSet = new List<Quad_hrtT>[len];
            QuadSet_Sum = new double[len];
            for (i = 0; i < len; i++)
                QuadSet[i] = new List<Quad_hrtT>();     //Init for AddToTC
            for (C = 0; C < len; C++)
            {
                #region Calculate tf-idf(C, e)
                List<IndexRelation> rel_c_set =
                    ProBaseInterface.FindTopInSup(Target_Class_Set[C], 1000);   //Top 1000 entities in C
                Dictionary<int, double> tfidf = new Dictionary<int, double>();
                int fc = ProBaseInterface.SupFreq(Target_Class_Set[C]);
                double denoValue = 0;
                foreach (IndexRelation ixrel in rel_c_set)
                {
                    int cls_count = ProBaseInterface.SubCount(ixrel.sub_idx);
                    double value = (1.0 * ixrel.freq / fc)
                        * Math.Log10(1.0 * MainFrame.TotalSupConcept / cls_count);
                    tfidf.Add(ixrel.sub_idx, value);            //The dictionary containing top entities in C
                    denoValue += value;         //Sum(tfidf(C,e))
                }
                #endregion
                for (T = 0; T < len; T++)
                {
                    if (T == C) Matrix[T, C] = 1;
                    else
                    {
                        List<IndexRelation> rel_t_set =
                            ProBaseInterface.FindTopInSup(Target_Class_Set[T], 1000);
                        double moleValue = 0;
                        foreach (IndexRelation ixrel in rel_t_set)
                        {
                            if (tfidf.ContainsKey(ixrel.sub_idx))
                                moleValue += tfidf[ixrel.sub_idx];
                            //Sum(tfidf(C,e)) e is in T and C
                        }
                        Matrix[T, C] = moleValue / denoValue;
                    }
                }
                if (C % 20 == 0)
                    OnProcessEvent(len, C, (int)(100.0 * C / len));
            }
            Relevant = new bool[len, len];
            int rele_cnt = 0;
            for (C = 0; C < len; C++)
                for (T = 0; T < len; T++)
                    if (Matrix[T, C] >= 0.1)
                    {
                        Relevant[T, C] = true;
                        rele_cnt++;
                    }
                    else
                        Relevant[T, C] = false;
            OnStateEvent("\tGenerating Matrix complete, with " 
                + rele_cnt.ToString() + " relevant pairs, have percentage of "
                + (rele_cnt * 100 / len / len).ToString() + "%.", true);
        }

        //put a Triple_hrt to Quad_hrtT
        public int AddToTC(Triple_hrt tp, int tuple_idx)
        { 
            int T, C, len = Target_Class_Set.Count;
            int h_idx = ProBase.SubConDict[tp.host_entity];
            int t_idx = ProBase.SubConDict[tp.target_entity];
            int f_t = ProBaseInterface.SubFreq(t_idx);
            int maxT = -1;
            double maxP = 0;
            for (T = 0; T < len; T++)       //traversal T
            { 
                double p = 0;
                int T_idx = ProBase.SupConDict[Target_Class_Set[T]];
                for (C = 0; C < len; C++)       //traversal C
                {
                    if (Relevant[T, C])     //p(T|C) must be large, that is Relevant[t,c] = 1
                    {
                        int C_idx = ProBase.SupConDict[Target_Class_Set[C]];
                        int f_ct = ProBaseInterface.OccurFreq(C_idx, t_idx);
                        p += 1.0 * f_ct / f_t * Matrix[T, C];
                        //p(c|t) = F(c,t) / F(t)
                        //p = Sigma(p(c,t))
                    }
                }
                Q_Class_Tuple[T, tuple_idx] = p;        //Add to record array
                if (p > maxP)
                { 
                    maxP = p;
                    maxT = T;
                }
            }
            TcForTuple[tuple_idx] = maxT;               //store the most likely class for this tuple
            if (maxT != -1) //Find the most likely target class
            { 
                int T_idx = ProBase.SupConDict[Target_Class_Set[maxT]];
                QuadSet[maxT].Add(new Quad_hrtT(h_idx, t_idx, T_idx, tp.fhrt, tp.q * maxP, tp.occPos));
                QuadSet_Sum[maxT] += tp.q * maxP;
            }
            return maxT;
        }

        public void GetTargetClass()        //if null, TcIdx = -1, Target_Class = "NULL"
        {
            int i, len = QuadSet_Sum.Length;
            int midx = -1;
            TcIdx = -1;
            Target_Class = "NULL";
            double maxP = 0;
            for (i = 0; i < len; i++)
                if (QuadSet_Sum[i] > maxP)
                {
                    midx = i;
                    maxP = QuadSet_Sum[i];
                }
            TcIdx = midx;
            if (TcIdx != -1)            //have target class
            {
                string tc = Target_Class_Set[midx];     //ht is the Target Class we found
                Target_Class = tc;
            }
        }
        #endregion

        #region Output Part
        int Compare_Idx(int a, int b)
        {
            double xa = Q_Class_Tuple[TcIdx, a];
            double xb = Q_Class_Tuple[TcIdx, b];
            return -(xa.CompareTo(xb));
        }

        public void OutputSeedPair(string hc, string attr, 
            int cur_case, string seedName)
        {
            int i, len = hrt_set.Count;
            HashSet<String> hosts = new HashSet<string>();
            List<Triple_hrt> OutputList = new List<Triple_hrt>();
            for (i = 0; i < len; i++)
            {
                Triple_hrt tp = new Triple_hrt(hrt_set[i]);
                tp.qhrt = tp.q;                             //qhrt stores q(h,r,t)
                tp.pct = Q_Class_Tuple[TcIdx, i];           //pct stores p(T|t)
                tp.q = tp.q * Q_Class_Tuple[TcIdx, i];      //q stores Q(h,r,t,T)
                OutputList.Add(tp);
            }
            OutputList.Sort();
            fsw = new FileStream(MainFrame.DataPath + seedName + ".txt", FileMode.OpenOrCreate);
            sw = new StreamWriter(fsw, Encoding.UTF8);
            sw.WriteLine("CONCEPTS ----------No.{0} : PAIR<\t{1}\t{2}\t{3}>----------",
                 cur_case, hc, attr, Target_Class);
            sw.WriteLine("host ent.\trelation\ttarget ent.\tf(h,r,t)\tq(h,r,t)\tp(T|t)\tQ(h,r,t,T)");
            for (i = 0; i < OutputList.Count; i++)
            {
                Triple_hrt tp = OutputList[i];
                if (!hosts.Contains(tp.host_entity) && tp.q != 0)
                {
                    hosts.Add(tp.host_entity);
                    sw.WriteLine("{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}", tp.host_entity,
                         tp.relation, tp.target_entity, tp.fhrt, tp.qhrt, tp.pct, tp.q);
                }
            }
            sw.Close();
            fsw.Close();
        }

        public void OutputSeedPair_Together(string hc, string attr,
            int cur_case, string seedName)
        {
            int i, len = hrt_set.Count;
            List<Triple_hrt> OutputList = new List<Triple_hrt>();
            for (i = 0; i < len; i++)
            { 
                Triple_hrt tp = new Triple_hrt(hrt_set[i]);
                tp.qhrt = tp.q;                             //qhrt stores q(h,r,t)
                tp.pct = Q_Class_Tuple[TcIdx, i];           //pct stores p(T|t)
                tp.q = tp.q * Q_Class_Tuple[TcIdx, i];      //q stores Q(h,r,t,T)
                OutputList.Add(tp);
            }
            OutputList.Sort();
            fsw = new FileStream(MainFrame.DataPath + seedName + "_Together.txt", FileMode.OpenOrCreate);
            sw = new StreamWriter(fsw, Encoding.UTF8); 
            sw.WriteLine("CONCEPTS ----------No.{0} : PAIR<\t{1}\t{2}\t{3}>----------",
                 cur_case, hc, attr, Target_Class);
            sw.WriteLine("host ent.\trelation\ttarget ent.\tf(h,r,t)\tq(h,r,t)\tp(T|t)\tQ(h,r,t,T)");
            for (i = 0; i < OutputList.Count; i++)
            {
                Triple_hrt tp = OutputList[i];
                sw.WriteLine("{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}", tp.host_entity,
                     tp.relation, tp.target_entity, tp.fhrt, tp.qhrt, tp.pct, tp.q);
            }
            sw.Close();
            fsw.Close();
        }

        //output seed pair by ranking the popular & relevant tuples
        public void OutputSeedPair_Popular_Relevant(string hc, string attr,
            int cur_case, string seedName)
        {
            int i, len = hrt_set.Count;
            List<Triple_hrt> OutputList = new List<Triple_hrt>();
            fsw = new FileStream(MainFrame.DataPath + seedName + "_PR.txt", FileMode.OpenOrCreate);
            sw = new StreamWriter(fsw, Encoding.UTF8);
            for (i = 0; i < len; i++)
            {
                if (TcForTuple[i] < 0)                          //filter tuples without target class
                {
                    sw.WriteLine("{0}\t{1}\t{2}\t{3}", hrt_set[i].host_entity, hrt_set[i].relation,
                        hrt_set[i].target_entity, "NULL");
                    continue;
                }
                if (Relevant[TcIdx, TcForTuple[i]] == false)     //filter those IRRELEVANT tuples
                {
                    sw.WriteLine("{0}\t{1}\t{2}\t{3}", hrt_set[i].host_entity, hrt_set[i].relation,
                        hrt_set[i].target_entity, Target_Class_Set[TcForTuple[i]]);
                    continue;
                }
                Triple_hrt tp = new Triple_hrt(hrt_set[i]);
                tp.q = //1.0 * (1 + Math.Log(tp.fhrt, 10))
                    1.0 * Math.Log(tp.fhrt, 2)
                    * (1.0 * tp.fhrt / tp.fhr)
                    * (1.0 * tp.fhrt / tp.frt);                 //We Need Popular!
                tp.blank_string = Target_Class_Set[TcForTuple[i]];          //Get own tc
                OutputList.Add(tp);
            }
            OutputList.Sort();
            sw.WriteLine("\n\n");
            sw.WriteLine("CONCEPTS ----------No.{0} : PAIR<\t{1}\t{2}\t{3}>----------",
                 cur_case, hc, attr, Target_Class);
            sw.WriteLine("host ent.\trelation\ttarget ent.\tf(h,r,t)\tf(h,r)\tf(r,t)\trank value\town tc");
            for (i = 0; i < OutputList.Count; i++)
            {
                Triple_hrt tp = OutputList[i];
                sw.WriteLine("{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}\t{7}", tp.host_entity,
                     tp.relation, tp.target_entity, tp.fhrt, tp.fhr, tp.frt, tp.q, tp.blank_string);
            }
            sw.Close();
            fsw.Close();
        }

        public void OutputAllHrtTc(string hc, string attr, string tc, int cur_case, string seedName)
        {
            int i;
            List<int> IdxOfTuple = new List<int>();         //对于找到的target class，把它的所有q(h,r,t,T)按降序排列的索引
            for (i = 0; i < hrt_set.Count; i++) IdxOfTuple.Add(i);
            IdxOfTuple.Sort(new Comparison<int>(Compare_Idx));  //desc order by P(T|t)

            fsw = new FileStream(MainFrame.DataPath + seedName + "_hrtT.txt", FileMode.OpenOrCreate);
            sw = new StreamWriter(fsw, Encoding.UTF8); 
            sw.WriteLine("CONCEPTS ----------No.{0} : PAIR<\t{1}\t{2}\t{3}>----------",
                 cur_case, hc, attr, Target_Class);
            sw.WriteLine("host ent.\trelation\ttarget ent.\tf(h,r,t)\tq(h,r,t)\tp(T|t)\tQ(h,r,t,T)");
            for (i = 0; i < IdxOfTuple.Count; i++)           //Then entities with Target Class
            {
                int idx = IdxOfTuple[i];
                Triple_hrt tp = hrt_set[idx];
                double pct = Q_Class_Tuple[TcIdx, idx];
                double qq = tp.q * Q_Class_Tuple[TcIdx, idx];
                sw.WriteLine("{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}", tp.host_entity, 
                    tp.relation, tp.target_entity, tp.fhrt, tp.q, pct, qq);
            }
            sw.WriteLine("ENDS");
            sw.Close();
            fsw.Close();
        }

        public void Output_Triple(List<Triple_hrt> hrt_set, string hc, string attr, int cur_case, string seedName)
        {
            fsw = new FileStream(MainFrame.DataPath + seedName + "_triple.txt", FileMode.OpenOrCreate);
            sw = new StreamWriter(fsw, Encoding.UTF8);
            sw.WriteLine("CONCEPTS ----------No.{0} : PAIR<\t{1}\t{2}\t{3}>----------",
                 cur_case, hc, attr, Target_Class);
            foreach (Triple_hrt t in hrt_set)
            {
                StringBuilder sb = new StringBuilder();
                sb.Append(t.host_entity).Append('\t')
                    .Append(t.relation).Append('\t')
                    .Append(t.target_entity).Append('\t')
                    .Append(t.fhrt).Append('\t')
                    .Append(t.fhr).Append('\t')
                    .Append(t.fht).Append('\t')
                    .Append(t.frt).Append('\t')
                    .Append(t.q);
                //.Append(t.q).Append('\t')
                //.Append(t.occPos);
                sw.WriteLine(sb.ToString());
            }
            sw.WriteLine("ENDS");
            sw.Close();
            fsw.Close();
        }

        #endregion

        public void GenerateSeedPair(string host_class, string attr, int cur_case, 
            Dictionary<string, int> SupConDict, string seedName, bool isDebug)
        {
            int TOP_R = 1000;               //only use top 1000 candidates
            int rd_cnt = 0;
            StreamWriter sw = null;
            FileStream fsw = null;
            hrt_set = new List<Triple_hrt>();
            Target_Class_Set = new List<string>();
            #region Read TOP_R candidates into hrt_set
            while (!sr.EndOfStream)
            {
                string rd = sr.ReadLine();
                rd_cnt++;
                if (rd == "ENDS")
                    break;
                if (rd_cnt <= TOP_R)
                {
                    string[] spt = rd.Split('\t');
                    Triple_hrt tp = new Triple_hrt();
                    tp.host_entity = spt[0];
                    tp.relation = spt[1];
                    tp.target_entity = spt[2];
                    tp.fhrt = int.Parse(spt[3]);
                    tp.fhr = int.Parse(spt[4]);
                    tp.fht = int.Parse(spt[5]);
                    tp.frt = int.Parse(spt[6]);
                    tp.q = double.Parse(spt[7]);
                    //tp.occPos = spt[8];
                    hrt_set.Add(tp);
                }
            }
            #endregion
            #region Generate Matrix
            int tp_cnt = 0, tot_cnt = hrt_set.Count, tc;
            GetTargetClassSet(attr, SupConDict);
            GenMatrix();
            #endregion
            #region Traversal Candidate
            if (isDebug)
            { 
                fsw = new FileStream(MainFrame.DataPath + seedName + "_TC.txt", FileMode.Append);
                sw = new StreamWriter(fsw, Encoding.UTF8);
                sw.WriteLine("host ent.\ttarget ent.\ttarget class\tP(T|t)\tQ(h,r,t,T)");
            }
            OnStateEvent("\tCalculating q<h,r,t,T>...", false);
            foreach (Triple_hrt tp in hrt_set)
            {
                tc = AddToTC(tp, tp_cnt);
                if (isDebug)
                {
                    if (tc != -1)
                    {
                        int len = QuadSet[tc].Count;
                        sw.WriteLine("{0}\t{1}\t{2}\t{3}\t{4}", tp.host_entity, tp.target_entity,
                            Target_Class_Set[tc], Q_Class_Tuple[tc, tp_cnt], QuadSet[tc][len - 1].qq);
                    }
                    else
                    {
                        sw.WriteLine("{0}\t{1}\t{2}\t{3}\t{4}", tp.host_entity, tp.target_entity,
                            "NULL", "NULL", "NULL");
                    }
                }
                if (++tp_cnt % 50 == 0)
                    OnProcessEvent(tot_cnt, tp_cnt, 
                        (int)(100.0 * tp_cnt / tot_cnt));
            }
            if (isDebug)
            {
                sw.Close();
                fsw.Close();
            }
            #endregion
            #region Output Seed Pair
            GetTargetClass();
            String dir = host_class + "_" + attr + "_" + Target_Class + @"\";
            Directory.CreateDirectory(MainFrame.DataPath + dir);
            seedName = dir + seedName;
            Output_Triple(hrt_set, host_class, attr, cur_case, seedName);       //Output all candidate tuples
            
            if (TcIdx != -1)    //really have the class
            {
                //OutputSeedPair(host_class, attr, cur_case, seedName);
                OutputSeedPair_Together(host_class, attr, cur_case, seedName);
                OutputSeedPair_Popular_Relevant(host_class, attr, cur_case, seedName);
                //if (isDebug)
                //    OutputAllHrtTc(host_class, attr, Target_Class, cur_case, seedName);
                //OutputSeedPair_PriorityTuple(host_class, attr, cur_case);
            }
            #endregion
        }

        public void Work(string filePath, 
            Dictionary<string, int> SupConDict, string seedName)
        {
            int cnt = 0;
            //string cur_seed = "";
            fsr = new FileStream(filePath, FileMode.Open);
            sr = new StreamReader(fsr, Encoding.UTF8);
            while (!sr.EndOfStream)
            {
                cnt++;
                string str = sr.ReadLine();
                string[] spt = str.Split('\t');
                string hc = spt[1];
                string attr = spt[2];
                OnStateEvent("Dealing No." + cnt.ToString() +
                    ":  <" + hc + "," + attr + ">", false);
                //cur_seed = seedName + "_" + attr;
                GenerateSeedPair(hc, attr, cnt, SupConDict, seedName, true);
                GC.Collect();
            }
            sr.Close();
            fsr.Close();
            OnStateEvent("Phase 3 complete.", true);
        }
    }
}
